{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0-beta1\n",
      "sys.version_info(major=3, minor=7, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.0.3\n",
      "numpy 1.16.2\n",
      "pandas 0.24.2\n",
      "sklearn 0.20.3\n",
      "tensorflow 2.0.0-beta1\n",
      "tensorflow.python.keras.api._v2.keras 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow import keras\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, tf, keras:\n",
    "    print(module.__name__, module.__version__)\n",
    "\n",
    "'''\n",
    "\n",
    "'''    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 28, 28) (5000,)\n",
      "(55000, 28, 28) (55000,)\n",
      "(10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "fashion_mnist = keras.datasets.fashion_mnist\n",
    "(x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()\n",
    "x_valid, x_train = x_train_all[:5000], x_train_all[5000:]\n",
    "y_valid, y_train = y_train_all[:5000], y_train_all[5000:]\n",
    "\n",
    "print(x_valid.shape, y_valid.shape)\n",
    "print(x_train.shape, y_train.shape)\n",
    "print(x_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x = (x - u) / std\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "# x_train: [None, 28, 28] -> [None, 784]\n",
    "x_train_scaled = scaler.fit_transform(\n",
    "    x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)\n",
    "x_valid_scaled = scaler.transform(\n",
    "    x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)\n",
    "x_test_scaled = scaler.transform(\n",
    "    x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tf.keras.models.Sequential()\n",
    "\n",
    "\"\"\"\n",
    "model = keras.models.Sequential()\n",
    "model.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model.add(keras.layers.Dense(300, activation=\"relu\"))\n",
    "model.add(keras.layers.Dense(100, activation=\"relu\"))\n",
    "model.add(keras.layers.Dense(10, activation=\"softmax\"))\n",
    "\"\"\"\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, activation='relu'),\n",
    "    keras.layers.Dense(100, activation='relu'),\n",
    "    keras.layers.Dense(10, activation='softmax')\n",
    "])\n",
    "\n",
    "# relu: y = max(0, x)\n",
    "# softmax: 将向量变成概率分布. x = [x1, x2, x3], \n",
    "#          y = [e^x1/sum, e^x2/sum, e^x3/sum], sum = e^x1 + e^x2 + e^x3\n",
    "\n",
    "# reason for sparse: y->index. y->one_hot->[] \n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer = \"sgd\",\n",
    "              metrics = [\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0714 18:19:09.061140 140736297124800 deprecation.py:323] From /Users/zhangyx/workspace/environments/tf2_py3/lib/python3.7/site-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 55000 samples, validate on 5000 samples\n",
      "Epoch 1/10\n",
      "55000/55000 [==============================] - 6s 118us/sample - loss: 0.5363 - accuracy: 0.8107 - val_loss: 0.4121 - val_accuracy: 0.8568\n",
      "Epoch 2/10\n",
      "55000/55000 [==============================] - 6s 114us/sample - loss: 0.3900 - accuracy: 0.8608 - val_loss: 0.3676 - val_accuracy: 0.8718\n",
      "Epoch 3/10\n",
      "55000/55000 [==============================] - 6s 109us/sample - loss: 0.3538 - accuracy: 0.8712 - val_loss: 0.3498 - val_accuracy: 0.8756\n",
      "Epoch 4/10\n",
      "55000/55000 [==============================] - 7s 120us/sample - loss: 0.3288 - accuracy: 0.8802 - val_loss: 0.3358 - val_accuracy: 0.8828\n",
      "Epoch 5/10\n",
      "55000/55000 [==============================] - 8s 139us/sample - loss: 0.3097 - accuracy: 0.8874 - val_loss: 0.3315 - val_accuracy: 0.8780\n",
      "Epoch 6/10\n",
      "55000/55000 [==============================] - 7s 122us/sample - loss: 0.2934 - accuracy: 0.8932 - val_loss: 0.3165 - val_accuracy: 0.8854\n",
      "Epoch 7/10\n",
      "55000/55000 [==============================] - 8s 152us/sample - loss: 0.2805 - accuracy: 0.8978 - val_loss: 0.3128 - val_accuracy: 0.8860\n",
      "Epoch 8/10\n",
      "55000/55000 [==============================] - 7s 129us/sample - loss: 0.2679 - accuracy: 0.9025 - val_loss: 0.3112 - val_accuracy: 0.8850\n",
      "Epoch 9/10\n",
      "55000/55000 [==============================] - 7s 132us/sample - loss: 0.2582 - accuracy: 0.9059 - val_loss: 0.3080 - val_accuracy: 0.8902\n",
      "Epoch 10/10\n",
      "55000/55000 [==============================] - 7s 127us/sample - loss: 0.2465 - accuracy: 0.9105 - val_loss: 0.3167 - val_accuracy: 0.8868\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(x_train_scaled, y_train, epochs=10,\n",
    "                    validation_data=(x_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAecAAAEzCAYAAAALosttAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd8HOWB//HPs129WHK3sWkG3DCmVxmOQO4ogQAOITkgBwQSTHqOkEYSUkm5XI4UQoAQIOBAuDiEhAs/LIhDCTaxwQ1jGxfZ2JasZpXVtuf3x+yuVl2yZe9o9X2/XvvamWeemX0eGfTVM8/sjLHWIiIiIu7hyXYDREREpCuFs4iIiMsonEVERFxG4SwiIuIyCmcRERGXUTiLiIi4zIDhbIy53xizxxizuo/txhjz38aYjcaYN4wxJwx/M0VEREaPwYycHwQu7Gf7e4Gjkq+bgJ8deLNERERGrwHD2Vr7IlDfT5VLgYes4xWg1BgzYbgaKCIiMtoMx5zzJGB7xnpNskxERET2g+9Qfpgx5iacU9+EQqH5U6dOPZQff1AkEgk8npF9XV0u9AFyox+50AdQP9wkF/oAudGPDRs21FlrKwdTdzjCeQcwJWN9crKsB2vtvcC9ADNmzLBvvfXWMHx8dlVXV1NVVZXtZhyQXOgD5EY/cqEPoH64SS70AXKjH8aYrYOtOxx/hiwB/j151fapQJO19t1hOK6IiMioNODI2RjzW6AKqDDG1ABfBfwA1tqfA88A/wpsBNqA6w9WY0VEREaDAcPZWnv1ANst8PFha5GIiMgoN7Jn10VERHKQwllERMRlFM4iIiIuo3AWERFxGYWziIiIyyicRUREXEbhLCIi4jIKZxEREZdROIuIiLiMwllERMRlFM4iIiIuo3AWERFxGYWziIiIyyicRUREXEbhLCIi4jIKZxEREZdROIuIiLiMwllERMRlfNlugIiIiOtZC/EoxDuc91gHxCOdr1hHxvYIxCJdt8cjQ/o4hbOIiGSPtZCIOcGWiEIi3rkcT64nohTu2ww1hQOEYi+h2VtQDrRPun7G9kT0kP5YFM4iIiORtf2HSXJbacMbsDHuBGA6BDPeE1GIxzLCMDZAWPZ2nNRyrEeodu4b6/l5qWMNwokAK4b4MzJe8AXB6wdvELwB8AWc99TLF4RAPnjLnHq+YNftfe0z0DF72/61CYNuusJZRKQvNgHRcN+jrn5CccAR2kCnQfvbPoSR3PEAq/aj78YDHr8TMh5vxnJyPbXs9SXLfBnhllxPlaXr+TKO09v21HF61lu9bgOzjp8/yFBMrnu8+9Fxd1A4i0j2pE9pDuZUY2r7EOb5+tw+uH2qEjF4YRj7azz9BEtqOQi+EIRKDnCk5pStfHMtx88/aeBQzXz3+MHjruuF6+qq4aiqbDfjkFE4i4xWibgTRLFwxquj63u0t/L2nvW61Z1btws2FfQStL2EL3Z4++VJjZx6OUWZGYCBwgFCz8+W7e8y7cijh376ssv2jOUsjOQaa7ww9dRD/rlyYBTOItmUmjeMtkG0nby2nbB7TT/B2EeQRvsIzP4C90AvcDFe8Oc5oeMLZbyH8CRiTlAFi4YYar2VBbueLu1v+zCP+LZUVzPtrKphO95IZmMxbCRCoqMDG4lioxFsJILt6HDeIxESHZH0so062xKp9cxtkYzySLTrMSLJ43c7dkUkwtsF+RifH+P1Ynw+8Pu6rBu/D3wZZfu97sN4neMZX0Ydn3fw6/6Mdvl84PcP6eetcBbpSyKeDs2e772V9betnzKbSH/kKQD/GEIbvYFuwRgEX0Zghkqdd3+ol3qhjFdGeTJwrfGTiIGNmeS1OxYbtSQicRLRhPOLtz1MItyObW93llvaseF2dmzdxoQJ450/PizJ9+QI2VoggbVhsOGu5dYCFpu5X1/bMsuTL0vm53TbntzJdm9TRtuc/Ts/t6ypiS333YfBOKFvDBiD8RjoUgbGJJeTZek66bKMOsaAx2CMcU5191WWWmeA43o8GWXJNiaPUbDlHfasWJEMvm4h2dGBjfZS3ktIkuj87/RAGL8fEwh0voJBTMAp8wSCzntxXnJbAE+yXvPOdymvrMTG49hYFGIxbDTWuR6NkWhrT67HIBZ1tse61ulcj0H00F6BPRQKZzmkbCxGIhx2fpmHwyTa27HhMIn2MDac/AXf3t65HG7HtoedfbqXtbWSaGtNHqudyo4ONhbk4fF5MV4PxmswPoPxJn/XeWzGK44xCYyJY0wMQwyPiWKIYWwEQwRDFONN1U/u7+1c9nh7KfcaTLDACTh/HvjzO9/zK3qWdau3buNWjp0zr//A9QWxngA2jjNSSf0s21I/t/aMn2fyZ9bQlv45JtrbsO0tJMJ12Pa2zp955r9Jezt2P35xmVAITyhE0Fpa3367M3RSAYZJVjQ9tkFye1/bTEYopV7pY/Wyb1/bunx+122m2zabHEGRSEAikQz25B8IiYQT5omMPw5swgn3RCJZluj8YyBZlq6TPEb6eKlX9+Om6pBx3ETCWe/nGKk/MAqB+lQgBoPpUPQEAxh/Z0h6SkuTy/50SPYMzwCmy7ZAZ3nq2P6M43c5RsAJ5v08s/FWdTXzq6r2a9++WGshHnfCOpoM9FR4Z67HYthYvMd6+o+EwaxHY3DbokG3TeEsgPMfqY1GM37Rt3UNzaEGaeo4GWWJtjaIDe5rE5mMF4zP4PGB8Vo83jgeTxzjtfh9Fo/PYgosBQVg42ATBhsz6eVEwji/qxJeZ1vCg02AjRvnPdZ9zjOQfO0njydjZODH+A0mEMMT6AB/Ao8/igm0YwItPUYRbXv2sPOf7STa2jOCNNxj2ba3D71dfj+evDw8oRAmL4QnLx9PKIQnPw9vWZlTnp+HJ5SHJy/PqRPKw5OflwzdjOW8fDx5IUyyzBMKYUKh9C/e6upqqob5F2k2vFNdzfEjuB/WWl6orqZqwYJsN8WVjDHp09gEgwf/AxXO2ZM6XWKjMWw0kvFXUzSjPNr5F1i38q5/aWWclsncNujyGDYWdT6vR3myHdEoleEw66PR/TptZfxePAEvxu/B4zMYH3h8Fo83gdcTx+OL4SmOYkqjeLwJJ1xTgZpa9nYuG6/FEwzgycvH5OfjKSjChIogUADBQghkLhc6c5qBAggUsmr9RubOPzVjJJrfOTr1hfqdi0zNpzlzZdHOU4DRaOepvdR6NNKjbtf6yXm0jG3d69pIhERLC4lopEv9YGsrbcXFTvClwrNiDP6MIDWhvK7LeSEnTDODMi8ZsKEQnvx8PMEgZohzXjLymcyzBTKiKJy7sYkE8aYm4vX1xOvridU3EG+oJ1ZfT7y+wSlrSC43NFDZ2spb1qYDb7jmZfpj/H7w+5MXQPjTFxyklvEa57SuJ/lVRQ+YkAWTwCTnogweDD4MlkgYQvkhPETwmAiGDjyEMcng9Hits+zLWE4GqjE4AZgMyF4DNF1e2HtZejl5DO/+hUjDnmo47PT9+5mm/nrOz9+v/YdLrow4ReTA5Hw421iMeGNjZ7hmBm1DMnzr653lvfXEGxv7DFhPYSHe8nJ8ZWX4J04kNHMmzXv3MnnatM6r/Pz+5FV7yeXeygNdAzV9daCJY+JhTMJ5EW/DxNsx8VbnPdoC0X2YyD4IN0PHPuhIvoebneVo28A/FI8PgsUQKoZgEY3t+ZSOndxHcBYkR6eZAVrUNUxH8Bf9RUTcaMSFs41EiDU0Em/IGNnW7+09cOvriTc3d1692Y23pARveTne8nIC06aRN+8EvOVl+MrL8ZaVdy6XlztzcoGe85AbqqsZd9YZneHYJSyT4Rlu6lxu3Qd7u9dLLtv4wD+AQJETlslgJVQKpVOd5WBxRugWd60XLOlc94W6nOpaqdGaiIirZD2cE+Fwl9PHnYGbcfo4Yzmxb1/vB/J48JaWJgN1DMEZM/CVl3UN2czl0lLnNOZQte6FzUth01LY8iJnNe+C6kE8bcQb6BaYxVB6WEZ4dtuWMbJNL2uUKiIyKmQtnH07drD+hPnYtj5Ow/p8+MrKkiPbMvImzkovp4LWV16WHvl6S0r2+xL9fsUiUPMP2PS889q5ErDOiPXwc9hRZJh65KyBQ9Z3CK4EFBGRnJC1cLahEGVXXtln4HqKipLfazzUDbNQvxk2/j8njLf8DSItzvd5ppwMC74IR5wLE48Hj5fN1dVMPafq0LdTRERyVtbCOT5mDOO+cHu2Pr6r9kZ454XO0XHjNqe8bDrMWQhHngfTznJGwCIiIgdZ1uecsyIeg52vd46Odyx3bqEYLIbpZ8MZn4QjFkD54dluqYiIjEKjJ5wbtsKmZBhvfhE6mpwvAU88Ac76rDM6njR/v79jKyIiMlxyN5w79sGWZZ2j4/pNTnnxZJh5qTNvPP0cyC/PbjtFRES6yZ1wTiTg3ZWd88bbX3Ue4u7Pd+aLT77JGR2POVK3sxMREVcb2eHctMP5zvHG/webq6G93imfMBdOX+SMjqecoq8xiYjIiDKywjnSBltfSo6O/x/UrnfKC8fD0Rc6YXx4FRRWZrOVIiIiB8Td4Wwt7F7deap668sQ73BuP3nY6TDvQ04gjz1Op6pFRCRnuC+cW/Y4t8ZMBXLrHqd87HFw8o1OGB92uvMYQBERkRyU/XCOhmH7K51hvOtNpzx/DBy+wLmI6/AFUDwhu+0UERE5RLIWzoFIIzx8hfN1p1g7ePww9VQ47ytwxHkwfo7zIGIREZFRZlDhbIy5EPgx4AXus9Z+p9v2qcCvgdJknduttc/0d8xgRx00bIET/t0ZHR92hvOMYBERkVFuwHA2xniBe4DzgRrgNWPMEmvt2oxqXwIWW2t/Zow5DngGmNbfcVsLp8Gi5fvbbhERkZw1mPPGJwMbrbWbrbUR4DHg0m51LJB6KkQJsHOggyZM9qe7RURE3MhYa/uvYMwVwIXW2huS6x8GTrHW3ppRZwLwf0AZUAD8i7V2RS/Hugm4CaCysnL+4sWLh6sfWdPS0kJh4cg+HZ8LfYDc6Ecu9AHUDzfJhT5AbvRjwYIFK6y1Jw6m7nANX68GHrTW/sAYcxrwG2PMLGttIrOStfZe4F6AGTNm2KqqqmH6+Oyprq5mpPcjF/oAudGPXOgDqB9ukgt9gNzpx2AN5rT2DmBKxvrkZFmm/wAWA1hrXwZCQMVwNFBERGS0GUw4vwYcZYyZbowJAB8AlnSrsw04D8AYcyxOONcOZ0NFRERGiwHD2VobA24FngXW4VyVvcYY83VjzCXJap8BbjTGrAJ+C1xnB5rMFhERkV4Nas45+Z3lZ7qVfSVjeS1wxvA2TUREZHTSLbhERERcRuEsIiLiMgpnERERl1E4i4iIuIzCWURExGUUziIiIi6jcBYREXEZhbOIiIjLKJxFRERcRuEsIiLiMgpnERERl1E4i4iIuIzCWURExGUUziIiIi6jcBYREXEZhbOIiIjLKJxFRERcRuEsIiLiMgpnERERl1E4i4iIuIzCWURExGUUziIiIi6jcBYREXEZhbOIiIjLKJxFRERcRuEsIiLiMgpnERERl1E4i4iIuIzCWURExGUUziIiIi6jcBYREXEZhbOIiIjLKJxFRERcRuEsIiLiMgpnERERl1E4i4iIuIzCWURExGUUziIiIi6jcBYREXEZhbOIiIjLKJxFRERcRuEsIiLiMgpnERERl1E4i4iIuIzCWURExGUGFc7GmAuNMW8ZYzYaY27vo85Vxpi1xpg1xphHh7eZIiIio4dvoArGGC9wD3A+UAO8ZoxZYq1dm1HnKOALwBnW2gZjzNiD1WAREZFcN5iR88nARmvtZmttBHgMuLRbnRuBe6y1DQDW2j3D20wREZHRYzDhPAnYnrFekyzLdDRwtDHm78aYV4wxFw5XA0VEREYbY63tv4IxVwAXWmtvSK5/GDjFWntrRp2ngShwFTAZeBGYba1t7Hasm4CbACorK+cvXrx4GLuSHS0tLRQWFma7GQckF/oAudGPXOgDqB9ukgt9gNzox4IFC1ZYa08cTN0B55yBHcCUjPXJybJMNcCr1too8I4xZgNwFPBaZiVr7b3AvQAzZsywVVVVg2mjq1VXVzPS+5ELfYDc6Ecu9AHUDzfJhT5A7vRjsAZzWvs14ChjzHRjTAD4ALCkW53/BaoAjDEVOKe5Nw9jO0VEREaNAcPZWhsDbgWeBdYBi621a4wxXzfGXJKs9iyw1xizFlgKfM5au/dgNVpERCSXDea0NtbaZ4BnupV9JWPZAp9OvkREROQA6A5hIiIiLqNwFhERcRmFs4iIiMsonEVERFxG4SwiIuIyCmcRERGXUTiLiIi4jMJZRETEZRTOIiIiLqNwFhERcRmFs4iIiMtkLZz3hi2RWCJbHy8iIuJaWQvnfRHLNfe9Qu2+jmw1QURExJWyFs6VeYY3dzRx8U+WsXJ7Y7aaISIi4jpZC+cCv+HJW07H5zVc9YuXWbx8e7aaIiIi4ipZvSBs5sQSltx6JidNK+PzT7zBV/+wmmhc89AiIjK6Zf1q7fKCAL++/mRuPGs6v355K9fc9yp1LZqHFhGR0Svr4Qzg83r44r8dx48/cDyrtjdy8U+W8UaN5qFFRGR0ckU4p1x6/CSevOV0PMZwxc9f5skVNdlukoiIyCHnqnAGmDWphD8uOpP5U8v4zO9WceeSNZqHFhGRUcV14QzOPPRv/uNk/uPM6Tz40hY+pHloEREZRVwZzuDMQ3/5ouP40cK5rNzeyCU/WcabNU3ZbpaIiMhB59pwTrls3mSevOV0jDFc8fOX+P3rmocWEZHc5vpwBmceesmtZzBvaimfXryKr/1R89AiIpK7RkQ4A4wpDPKb/ziF68+YxgN/38K//+of7NU8tIiI5KARE84Afq+Hr148kx9eNZcV2xq45H/+zuodmocWEZHcMqLCOeXyEybzxM2nkbCW9//sJf73nzuy3SQREZFhMyLDGWDO5FL+uOhM5k4p5ZOPr+QbT68lpnloERHJASM2nAEqCoM8csMpXHf6NH617B3+/f5/UN8ayXazREREDsiIDmdw5qHvvGQmd18xh+VbG7j4J8tYs1Pz0CIiMnKN+HBOufLEKfzuo53z0H9YqXloEREZmXImnAHmTillya1nMmdSKZ94bCXf/JPmoUVEZOTJqXAGqCwK8siNp3DtaYfxy7+9w3UPvEaD5qFFRGQEyblwBmce+muXzuJ7V8zhH+/Uc/H/LGPtzuZsN0tERGRQcjKcU646cQqLbz6NWNxy+c/+zpJVO7PdJBERkQHldDgDHD+llCWLzmD2pBJu++0/+fYz64gnbLabJSIi0qecD2eAsUUhHrnhVD586mH84sXNXPfAP2hs0zy0iIi406gIZ4CAz8M33jeL775/Nq9uduah172reWgREXGfURPOKQtPmspjHz2VSCzB5T99iaff0Dy0iIi4y6gLZ4ATppbxx0VnctzEYm599J9858/rNQ8tIiKuMSrDGZx56N/eeCrXnDKVn7+wiesffE3z0CIi4gqjNpzBmYf+5mWz+fbls3l5Ux2X/M/fWb9L89AiIpJdozqcU64+eSqP3XQa4Wicy3/6Es+8+W62myQiIqOYwjlp/mFlPL3oTI4ZX8THHnmd7/1F89AiIpIdCucMY4tD/PamU7n65Kn8tHoTH3nwNZraotluloiIjDKDCmdjzIXGmLeMMRuNMbf3U+/9xhhrjDlxoGPujO7ki8u+yFNvP8X25u1Y645RatDn5duXz+Zbl83mpU11XHLPMt7atS/bzRIRkVHEN1AFY4wXuAc4H6gBXjPGLLHWru1Wrwj4BPDqYD44YAL8reZvLNm0BICxeWOZP34+J447kfnj5nN4yeEYY4bYneHzwVOmMmN8ITc//DqX/fTv/ODKubx39oSstUdEREaPAcMZOBnYaK3dDGCMeQy4FFjbrd43gO8CnxvMB1f4Knhh4QtsbtrMit0rWL5rOct3LefP7/wZgLJgGfPHzWf+uPmcOP5Ejio9Cq/HO9h+DYv5h5Xz9KIzufnhFdzyyOvcuuBIPnX+0Xg92fujQUREct9gwnkSsD1jvQY4JbOCMeYEYIq19k/GmEGFc3I/jig9giNKj+CqGVdhrWX7vu1OWO9ezordK3hu23MAFPmLOGHcCenAPnbMsfg9/sF+1H4bVxzisZtO5at/WMP/LN3Imp1N/NcH5lGSd/A/W0RERicz0FyvMeYK4EJr7Q3J9Q8Dp1hrb02ue4DngeustVuMMdXAZ621y3s51k3ATQCVlZXzFy9ePGAD62P1bApvYmPHRjaGN7IntgdwTosfHjycI0NHckTwCA4LHobfHLzAtNaydHuMR9ZFqMgz3HZCiEmFHlpaWigsLDxon3so5EIfIDf6kQt9APXDTXKhD5Ab/ViwYMEKa+2A12TB4ML5NOBOa+0FyfUvAFhrv51cLwE2AS3JXcYD9cAlvQV0yowZM+xbb701mDZ2Uddelz4NvmLPCt5ueBuAgCfAnMo56dPgcyrmkO/PH/LxB7J8Sz03P/w67ZEYP7jqeEJ166mqqhr2zzmUqqurR3wfIDf6kQt9APXDTXKhD5Ab/TDGDDqcB3Na+zXgKGPMdGAH8AHgg6mN1tomoCLjw6vpY+Q8HCryKrhg2gVcMO0CAJo6mlixe0X6VPgv3/wlv3jjF/iMj+MqjktfYDZv7DyKAkUH/PknTuuch7754RVMKfJwau0qZk8uYdakEo6bUEzIf2jnxkVEJLcMGM7W2pgx5lbgWcAL3G+tXWOM+Tqw3Fq75GA3sj8lwRLOnXou5049F4CWSAsra1c6I+vdK3ho7UPcv/p+PMbDjLIZ6ZH1CWNPoCxUtl+fOb4kxOMfPZX7/vYOf1mxkefX7+F3K2oA8HoMR40tZObEEmZPKmb25BKOm1BCXkCBLSIigzOYkTPW2meAZ7qVfaWPulUH3qz9Vxgo5MxJZ3LmpDMBaI+180btG+mR9e82/I6H1z0MwJGlRzphnRxdV+ZXDvpzgj4vH19wJDNNDeeccw7vNoV5c0cTq5OvFzbU8uTrTmB7DBw5tpBZE53RtRPYxRQEB/XjFxGRUSbn0yHPl8cpE07hlAnOBeaReITVdavTYb1k0xIef+txAA4rPqxLWE8snDiozzDGMLE0j4mleVwwczzgXEC2u7mDN3c08eaOJtbsaGLZxjp+/88dyX3giMpCZk0sdgJ7UgkzJ5VQqMAWERn1Rl0SBLwBThh3AieMO4EbuZFYIsb6+vXp0+B/3fpXfv/27wGYUDAhHdQnjj+RqUVTB31jFGMM40tCjC8Jcf5x49Lle5rD6cBevaOJVzbX878rdyb3geljCtJhPWtSCTMnFVMc0te2RERGk1EXzt35PD5mVcxiVsUsrpt1HfFEnI2NG9Pfs/77zr/zx81/BJyL0TJH1keUHjHkzxtbHOK84hDnHdsZ2LX7OlidEdjLt9SzZNXO9PZpY/LTgZ0aYet71iIiuWvUh3N3Xo+XGeUzmFE+g2uOvQZrLe80vZMO6+W7l/PslmcB52K0ElvCk88/SWVeJZV5lVTkV3Qu51UwJm8MPk//P+bKoiALjhnLgmPGpsvqWjrS89erdzTzz22NPP1G56Msp5bnp0fXznsxpfmBg/NDERGRQ0rhPABjDIeXHs7hpYen72JW01LD8l3LWVW7ijXb17CjZQer9qyioaOh5/4YykJlvQZ3ZX7X5aA3mN6vojBI1YyxVM3oDOz61kh6hL1mZxNv7GjkTxnPnp5cltctsEsoL1Bgi4iMNArnITLGMKVoClOKpnDZUZd1+WJ8NB5lb3gvtW211LbXUtdeR217LbVtnctv17/N3vBe4jbe49hFgaIeI/CKvGSg5zvLJ0yr5KyjKtJz341tEVbvaHZOie90Rtp/Xr0rfcxJpXnMmlScDutZk0qoKAz2+GwREXEPhfMw8nv9jC8Yz/iC8f3WiyfiNHQ0OIHd1nuIr9yzktq2WiKJSI/983x56dBOjborxlRw/uQKPphXScgzlrrGIO/stqx5t4XVO5p4ds3u9P4TSkJd5rAbwwmstVl9CpiIiHRSOGeB1+OlIq+CirwKjik/ps961lqaI829hnddm/O+oWEDf9/5d1qjrT3293l8jAmNYfwxlRwTHIMnUUxHuJDGlhBr6/08/06ARLQIGyvkjr8/y9TyfKaOyeew8nwOG5PP1DEFHFaez6SyPPzeQT36W0REhoHC2cWMMc5FZ8GSAa8Mb4u2dYZ4RnjXtddR117H7vad1LW90TkvXgz5xZ37+00Be20BtdF8XtoWIr4pHxsvwMbzIVFAWaiUCYUVTCmp4PDysRxdOY7DK4qZWp6vm6mIiAwz/VbNEfn+fKb6pzK1eGq/9XqbF1++bjmlE0ppDDfS2NFIQ0cDe9t30NTRRCQRBqAN5+kmm1qhuhXYDjYewsYL8NpC8n3FFPtLGJNfxvjCMUwurmR6eSVTSiooD5Wn/8g4FI/5FBEZ6RTOo0xv8+Jj3x1L1SlVvdZvj7XT1NHkhHa4gcaORt7dt5dtjbXs3FdHbVsDjeEGWmKN7IrUsDPWwuqWKOzq9XAEPQUUB0ooD5UxtmAMZaFSSoOllIXKKAmWUBYsozTolJWGShXoIjIqKZylX3m+PPJ8eQNe5JYSjsbZVNfAW3t2saFuN1sba9nZXMeetnoaw41EPK20eFt519vGOt8mfP428LSSMD0vfEsp8hdRGiqlLJgM8FDXAE+Vb+vYxoaGDfg9/s6X199l3evRA0hExP0UzjKsQn4vMydUMHNCBTCry7Z4wrKrOczWva1s29vG1vq25HsrW+ubaIk2Y7ytGG8bxttKcWGEkoIo+b4w/lg7bbaV5o7dbGh4m+ZIE+2x9p4NGOAZaR7j6RreGQHu8/jSZT6Pr0ew91jvrSxjvddj9LJP+nO9fsKJsK6cFxGFsxw6Xo9hUmkek0rzOL3b9W3WWhrbomytb+sR3lu2tLJnX0eX+sUhH1PH+JlQnqCiOEZpYYS63e8w59jpFIQMXm+cWCJGNBElGo8675mvZFl/dTpiHbQkWnrsk3rFEjGi8SgxGxvWn9MXHv4C5cFyykJl6Vd5qJyyYMZy8r08VE4pV8v1AAAezElEQVRRoAiP0dX0IrlE4SyuYIyhrCBAWUGA46eU9tjeHomzLRXc9W1sTYb3xp2tVK+JEEt4gSN5YqVTPz/gZ3xxiHHFzsNHxhWHGF8c5LDiEONKQowvDlFZFByWr4glbIJYItYZ9L2F/UDrybK1G9YyZvIYGsIN1IfraQg3ULOvhoaOhl6/LgfgNd70vH0quMuCGcvdwr00WKrT+yIup3CWESEv4GXG+CJmjC/qsS0WT/BuU5hnql9m/OHHsLs5zK6mDue9OcxrW+rZ09xBJJ7osp8xzm1SO0PcWR5b7IR3KtSLQ75+TzN7jIeAN0DAe+C3Sq3eXU3V/Kpet0XikXRgN4QbqO/IWE6VdzTwVv1b1IfraY4093ocg0mHeffg7i3gS0OluihP5BBTOMuI5/N6mFKez4xyL1XHT+q1TiJhaWiLsKs5nA7vXc1hdjc5AV7T0MaKrfU0tEV77Jvn9yaDujPIu4zIS0KMHaZReH8C3sCg7kCXEk1EaepoSgd3fbi+S7g3dDhlmxo3pa/Et9hej1UUKEqfRu9+er0sVNblNHx5qHw4uy2SE6zt/f+tviicZVTweAxjCoOMKQwyc2JJn/XC0Th7mp3gzgzvXc1h9jSHWbGtgd1NvY/CxxQEe46+M06jjy8OUZzX/yh8OPk9/vSd6AYjnojTFGnqOhLvNkJvCDewvWU7b9S9QWO4sc/5dh8+Ao8E8Hl86QveMt99Hh8+41wwl1pOl2fs071+9+P1tp5eNgMcr9tx/V6/06ZkHV2UN3pZa2mPtdMWa6M96ry3xdpojbbSFnWW+3tvjbam988sHwqFs0iGkN/L1DHObUz7Yq2loS3KrqZw+tR5anl3c5gdjWFe39ZIfWvPr4eF/J7eR9/J0+p17QnC0Tgh/6GfE/Z6vOnR8REM/Kzy1O1lewvydZvWMXHyxPSFc11e1rmQLmo7t7XF29Jz9r3Wz7h4L5YY3gvw+uI1Xow1+B52gtpjPJ0vOpeNMXiN11nG4PV4MXSr38t+mfsaY5xtnp51MvftUt94uu7byz7GGGoaalizcg1e403/MeL1OMvpsuSy1+PFb/z9bs/8o2eg7YfiQsWDFaR9nUXqzmu85Pvzyffld3kflz+OPH9el/JFLBp0vxTOIkNkjKG8IEB5QYDjJhb3Wa8jljEKTwV5U5jd+zrY3RRm5fZGdq0JE4l1HYV/9oW/UBT0MaYwQEVh0HkVOctjCoNUdikPUhDwZmWUl3l72ekl07tsq66vpuqkqoPyudZa4jbe5SK8zDBPhX6vQd/twr1U+Pd1rHe2vsOUKVNI2ARxG8diSdhEry+L066ETaTbaG1GfbrVz+hHat8u9VP7JJz3VP3ux89sV2/HiCVi/HXVXw/Kv8VAUn+s+D3+LuHdoyz1R0My7LuUJffZVbeLB/78wIEHaSosM4J0bP7YXgO2wFeQXk8FbYG/oEudgCcw6P//FM4iLhD0eZlSns+U8v5H4Y1t0fSp8xdfW0XFpOnU7uugrqWDvS0RNtW28Oo7Hb3Ohzuf40kHdWVhgDEFnWHe+XLWS/P9I/50rTEmPTo72Kqbq6k6seqgf87BVF1dzTnnnOMEtY0RT8SJJqLEbZx4Ip7+AyW1nP7DJ6NsoO2ZfywNdnv3sngiTtRG0/tE4hHaE+3OH0w2Rke8gwJPgROkvQRsOjgzgjTf31k21CDNNoWzSBZlfoXs2AnFmHf9VFUd2WvdaDxBfWuEupYO6loi1CUDPBXitS0d7GgMs6qmifrWCPFEz9GEz2PSI/IxydCuzBidjynoXC7PD+DT08hyQup0uBcvjNBv0VVXV1NVVZXtZhwyCmeREcLv9aTnqgeSSFga26NOeO/roDYZ4Kkwr0sub9rTQm1LR49T6+Bc5FaWH0iPulOvMalAzxidjykMEPSN0N/6Ii6kcBbJQR5P57z40eN6fjc8k7WWfR2x5Eg8wt5kgNemwnxfB3tbI6yqaaRuXwetkXivxykK+dKj8Hh7mL/sfYOSfD+leQHK8v2U5vspyQtQmlwuzQuQF1Cgi/RG4SwyyhljKA75KQ75Obxy4PrtkXiPEXj6FHurc7r93dYE29bvoakt2uNrZ5mCPk86qJ0gTwZ3fiBdXposL0mWl+X7yfNn5yI4kUNF4SwiQ5IXGPhCt9T8oLWW9micxrao82qP0NQWpSFjOVXe2BZlW30bq2oiNLRFez3VnhLwerqEeWpEXpYM8JI8f5dwL8nzU1YQyNqV7SJDpXAWkYPGGEN+wEd+wMfE0rwh7RtOhXoyuJ1XhMZ2Z7kpo3xHYztrdjbR2BalPdr7aXdwLohLhXVpfiA9Ii9LLpfm+ynJWN7TlqCpPUpR0IfHo1CXQ0fhLCKuFPJ7GV/i3Dp1KMLROM3t0XSIN7ZFuoZ8e9QZsbc7t3Ndv2sfjW2RPufSP//i/+ExUJyXPL2e1zXAS/I6w75zxJ6q59eFcrJfFM4iklNCfi8hv5exg7iqPVMk5oySUyPyhrYor/7zDcZPPSJZHu0M9/Yo2/a20tgepbk9Si/fWkvL83vTod0Z5J1hnnkKPl0v309hQKP10UzhLCICBHweKouCVBYF02X+PX6qzjq83/0SCedq96a2ZIAnw70z0CNdgn1bfRtv1Dj1wtG+59U9hvSIPDVqT5+ST47eS7qVlyTfNVof+RTOIiIHwOMx6RHwUHU/BZ8Z5t1H6o1tEbbsbU1v6+8hR3l+bzqwbUc7v9nyGoUhH4XBzldB0EdhyEdRxnJ6e8hHQcCHVyP3rFE4i4hkyf6egk8kLPvCsR4jdWc+vdtI/d1WdjWHaamN0doRY184Rkc/V8Jnyg94ewR2n4GeLCsKJbdlLOf7vTpFP0QKZxGREcbjMc4p7Hw/U+n7K22Q+lrbWV3KovFEOqhbIzFawjFaOpKvbsutkWS9ZNn2+rYu+0TjAz90whgoDHQGekHQCfiuge6lMOhPBr6zXBD0UpQsa+6wRGIJAr7RcUtZhbOIyCjj93qSN3oJHPCxOmJxJ8Q74uzriNLaEaelI5oMdGe5pSOeDPNosp4T9rX7Ojr/EOiI9Xo/+C6W/pk8v5fiPB8lec6Nc0ry/BTn+SkO+TqXu2zrLB9JF9m5Kpyj0Sg1NTWEw+FsN2XQSkpKWLduXbabcUD66kMoFGLy5Mn4/UOfSxOR0SHo8xIs9DKm8MCOY60lHE2kgzo1snfWo6x4Yx3jp0yjqT1Kc7tzSr85HGX3vjAb9uyjuT1Gc7j/uXiPgaKQv2e4h7qGeGeZn5I8XzrsD+Vz1l0VzjU1NRQVFTFt2rQRcxefffv2UVTU/72L3a63Plhr2bt3LzU1NUyfPr2PPUVEhocxhryAl7yAt8sV8yllTRupqjqq32MkEpaWiHPlfHM4mg7y5mSQNycvpmtOztc3t0fZuKclXbe/q+fBuaK/pNsovaTbKD1zNJ+5rTA0tLh1VTiHw+ERFcy5zBjDmDFjqK2tzXZTREQGxePpvE/8/uiIxdMj8N6CPBXyqdDf2xLhnbrWZPkgTssPgavCGVAwu4j+LURkNAn6vFQW9T5yH4i1ltZIPB3kXd6TAf+Z7w7+eK4L52wrLCykpaUl280QEZERxBiT/krZpD7uI/+ZIRxvdFyTLiIiMoIonPtgreVzn/scs2bNYvbs2Tz++OMAvPvuu5x99tkcf/zxzJo1i5deeol4PM51112XrvujH/0oy60XEZGRzLWntb/2xzWs3dk8rMc8bmIxX7145qDq/v73v2flypWsWrWKuro6TjrpJM4++2weffRRLrjgAr74xS8Sj8fZvXs3K1euZMeOHaxevRqAxsbGYW23iIiMLq4N52xbtmwZV199NV6vl3HjxnHOOefw2muvcdJJJ/GRj3yEaDTK+973Po444gjy8vLYvHkzixYt4t/+7d94z3vek+3mi4jICObacB7sCPdQO/vss3nxxRf505/+xHXXXcctt9zCRz/6UVatWsWzzz7Lz3/+cxYvXsz999+f7aaKiMgIpTnnPpx11lk8/vjjxONxamtrefHFFzn55JPZunUr48aN48Ybb+SGG25In/ZOJBK8//3v56677uL111/PdvNFRGQEc+3IOdsuu+wyXn75ZebOnYsxhu9973uMHz+eX//619x99934/X4KCwv56U9/yo4dO7j++utJJJy7y3z729/OcutFRGQkG1Q4G2MuBH4MeIH7rLXf6bb908ANQAyoBT5ird06zG09JFLfcTbGcPfdd3P33Xd32X7ttddy7bXXptdTt77UaFlERIbLgKe1jTFe4B7gvcBxwNXGmOO6VfsncKK1dg7wBPC94W6oiIjIaDGYOeeTgY3W2s3W2gjwGHBpZgVr7VJrbVty9RVg8vA2U0REZPQwtr/nawHGmCuAC621NyTXPwycYq29tY/6/wPsstbe1cu2m4CbACorK+cvXry4y/aSkhKOPPLI/elH1sTjcbzeQ/cYsYOhvz5s3LiRpqamQ9yi/dPS0kJh4QE+ty7LcqEPoH64SS70AXKjHwsWLFhhrT1xMHWH9YIwY8yHgBOBc3rbbq29F7gXYMaMGbaqqqrL9nXr1o24xy/m6iMjU0KhEPPmzTvELdo/1dXVdP9vaqTJhT6A+uEmudAHyJ1+DNZgwnkHMCVjfXKyrAtjzL8AXwTOsdZ2DE/zRERERp/BzDm/BhxljJlujAkAHwCWZFYwxswDfgFcYq3dM/zNFBERGT0GDGdrbQy4FXgWWAcsttauMcZ83RhzSbLa3UAh8DtjzEpjzJI+DiciIiIDGNScs7X2GeCZbmVfyVj+l2FuV86LxWL4fLoHjIiI9KTbd/bife97H/Pnz2fmzJnce++9APzlL3/hhBNOYO7cuZx33nmAc/XgLbfcwuzZs5kzZw5PPvkkQJcrCp944gmuu+46AK677jpuvvlmTjnlFD7/+c/zj3/8g9NOO4158+Zx+umn89ZbbwHO1dOf/exnmTVrFnPmzOEnP/kJzz//PO973/vSx/3rX//KZZdddih+HCIicoi5d+j259th15vDe8zxs+G93xmw2v333095eTnt7e2cdNJJXHrppdx44428+OKLTJ8+nfr6egC+8Y1vUFxczJtvOu1saGgY8Ng1NTW89NJLeL1empub+dvf/obP5+O5557jjjvu4Mknn+Tee+9ly5YtrFy5Ep/PR319PWVlZXzsYx+jtraWyspKHnjgAT7ykY8c2M9DRERcyb3hnEX//d//zVNPPQXA9u3buffeezn77LOZPn06AOXl5QA899xz3Hfffen9ysrKBjz2lVdemf5OcVNTE9deey1vv/02xhii0Wj6uDfffHP6tHfq8z784Q/z8MMPc/311/Pyyy/z0EMPDVOPRUTETdwbzoMY4R4M1dXVPPfcc7z88svk5+dTVVXF8ccfz/r16wd9DGNMejkcDnfZVlBQkF7+8pe/zIIFC3jqqafYsmXLgN/hu/7667n44osJhUJceeWVmrMWEclRmnPupqmpibKyMvLz81m/fj2vvPIK4XCYF198kXfeeQcgfVr7/PPP55e//GV639Rp7XHjxrFu3ToSiUR6BN7XZ02aNAmABx98MF1+/vnn84tf/IJYLNbl8yZOnMjEiRO56667uP7664ev0yIi4ioK524uvPBCYrEYxx57LLfffjunnnoqlZWV3HvvvVx++eXMnTuXhQsXAvClL32JxsZGZs2axdy5c1m6dCkA3/nOd7jooos4/fTTmTBhQp+f9fnPf54vfOELzJs3Lx3EADfccANTp05lzpw5zJ07l0cffTS97ZprrmHKlCkce+yxB+knICIi2abzot0Eg0H+/Oc/97rtve99b5f1wsJCfvGLX/S49eUVV1zBFVdc0WP/zNExwGmnncaGDRvS63fd5dyO3Ofz8cMf/pAf/vCHPY6xbNkybrzxxkH1RURERiaF8wgyf/58CgoK+MEPfpDtpoiIyEGkcB5BVqxYke0miIjIIaA5ZxEREZdROIuIiLiMwllERMRlFM4iIiIuo3AWERFxGYXzAch8+lR3W7ZsYdasWYewNSIikisUziIiIi7j2u85f/cf32V9/eAfNjEYx5Qfw3+e/J99br/99tuZMmUKH//4xwG488478fl8LF26lIaGBqLRKHfddReXXnrpkD43HA5zyy23sHz58vTdvxYsWMCaNWu4/vrriUQiJBIJnnzySSZOnMhVV11FTU0N8XicL3/5y+nbhYqIyOjg2nDOhoULF/LJT34yHc6LFy/m2Wef5bbbbqO4uJi6ujpOPfVULrnkki5PnhrIPffcgzGGN998k/Xr1/Oe97yHDRs28POf/5xPfOITXHPNNUQiEeLxOM888wwTJ07kT3/6E+A8HENEREYX14ZzfyPcg2XevHns2bOHnTt3UltbS1lZGePHj+dTn/oUL774Ih6Phx07drB7927Gjx8/6OMuW7aMRYsWAXDMMcdw2GGHsWHDBk477TS++c1vUlNTw+WXX85RRx3F7Nmz+cxnPsN//ud/ctFFF3HWWWcdrO6KiIhLac65myuvvJInnniCxx9/nIULF/LII49QW1vLihUrWLlyJePGjevxjOb99cEPfpAlS5aQl5fHv/7rv/L8889z9NFH8/rrrzN79my+9KUv8fWvf31YPktEREYO146cs2XhwoXceOON1NXV8cILL7B48WLGjh2L3+9n6dKlbN26dcjHPOuss3jkkUc499xz2bBhA9u2bWPGjBls3ryZww8/nNtuu41t27bxxhtvcMwxx1BeXs6HPvQhSktLue+++w5CL0VExM0Uzt3MnDmTffv2MWnSJCZMmMA111zDxRdfzOzZsznxxBM55phjhnzMj33sY9xyyy3Mnj0bn8/Hgw8+SDAYZPHixfzmN7/B7/czfvx47rjjDl577TU+97nP4fF48Pv9/OxnPzsIvRQRETdTOPfizTffTC9XVFTw8ssv91qvpaWFffv29bpt2rRprF69GoBQKMQDDzzQo87tt9/O7bff3qXsggsu4IILLtjfpouISA7QnLOIiIjLaOR8gNasWcPNN9/cpSwYDPLqq69mqUUiIjLSKZwP0MyZM1m5cmW2myEiIjlEp7VFRERcRuEsIiLiMgpnERERl1E4i4iIuIzC+QD09zxnERGR/aVwzgGxWCzbTRARkWHk2q9S7frWt+hYN7zPcw4eewzj77ijz+3D+TznlpYWLr300l73e+ihh/j+97+PMYY5c+bwm9/8ht27d3PzzTezefNmAH72s58xceJELrroovSdxr7//e/T0tLCnXfeSVVVFccffzzLli3j6quv5uijj+auu+4iEokwZswYHnnkEcaNG0dLSwuLFi1i+fLlGGP46le/SlNTE2+88Qb/9V//BcCDDz7I5s2b+dGPfnRAP18RERkerg3nbBjO5zmHQiGeeuqpHvutXbuWu+66i5deeomKigrq6+sBuO222zjnnHN46qmniMfjtLS00NDQ0O9nRCIRli9fDkBDQwOvvPIKxhjuu+8+vve97/GDH/yAb3zjG5SUlKRvSdrQ0IDf7+eb3/wmd999N36/n4cfflgP2BARcRHXhnN/I9yDZTif52yt5Y477uix3/PPP8+VV15JRUUFAOXl5QA8//zzPPTQQwB4vV5KSkoGDOeFCxeml2tqali4cCHvvvsukUiE6dOnA/Dcc8/x2GOPpeuVlZUBcO655/L0009z7LHHEo1GmT179hB/WiIicrC4NpyzJfU85127dvV4nrPf72fatGmDep7z/u6XyefzkUgk0uvd9y8oKEgvL1q0iE9/+tNccsklVFdXc+edd/Z77BtuuIFvfetbHHPMMXzoQx8aUrtEROTg0gVh3SxcuJDHHnuMJ554giuvvJKmpqb9ep5zX/ude+65/O53v2Pv3r0A6dPa5513XvrxkPF4nKamJsaNG8eePXvYu3cvHR0dPP300/1+3qRJkwD49a9/nS4///zzueeee9LrqdH4Kaecwvbt23n00Ue54oorBvvjERGRQ0Dh3E1vz3Nevnw5s2fP5qGHHhr085z72m/mzJl88Ytf5JxzzmHu3Ll8+tOfBuDHP/4xS5cuZfbs2cyfP5+1a9fi9/v5yle+wsknn8z555/f72ffeeedXHnllcyfPz99yhzgS1/6Eg0NDcyaNYu5c+eydOnS9LarrrqKM844I32qW0RE3EGntXsxHM9z7m+/a6+9lmuvvbZL2bhx4/jDH/7Qo+5tt93Gbbfd1qO8urq6y/qll17a61XkhYWFXUbSmZYtW8anPvWpXreJiEj2aOQ8CjU2NnL00UeTl5fHeeedl+3miIhINxo5H6CR+Dzn0tJSNmzYkO1miIhIHxTOB0jPcxYRkeHmutPa1tpsN0GS9G8hIpIdrgrnUCjE3r17FQouYK1l7969hEKhbDdFRGTUcdVp7cmTJ1NTU0NtbW22mzJo4XB4xAdYX30IhUJMnjw5Cy0SERndBhXOxpgLgR8DXuA+a+13um0PAg8B84G9wEJr7ZahNsbv96dvOzlSVFdXM2/evGw344DkQh9ERHLJgKe1jTFe4B7gvcBxwNXGmOO6VfsPoMFaeyTwI+C7w91QERGR0WIwc84nAxuttZuttRHgMaD73S4uBVJ3ungCOM8M9NgmERER6dVgwnkSsD1jvSZZ1msda20MaALGDEcDRURERptDekGYMeYm4KbkaocxZvWh/PyDpAKoy3YjDlAu9AFyox+50AdQP9wkF/oAudGPGYOtOJhw3gFMyVifnCzrrU6NMcYHlOBcGNaFtfZe4F4AY8xya+2Jg22oW+VCP3KhD5Ab/ciFPoD64Sa50AfIjX4YY5YPtu5gTmu/BhxljJlujAkAHwCWdKuzBEg9yeEK4HmrLyuLiIjslwFHztbamDHmVuBZnK9S3W+tXWOM+Tqw3Fq7BPgV8BtjzEagHifARUREZD8Mas7ZWvsM8Ey3sq9kLIeBK4f42fcOsb5b5UI/cqEPkBv9yIU+gPrhJrnQB8iNfgy6D0Znn0VERNzFVffWFhERkSyFszHmQmPMW8aYjcaY27PRhgNljLnfGLNnJH8dzBgzxRiz1Biz1hizxhjziWy3aaiMMSFjzD+MMauSffhattt0IIwxXmPMP40xT2e7LfvLGLPFGPOmMWblUK5OdRNjTKkx5gljzHpjzDpjzGnZbtNQGWNmJP8NUq9mY8wns92uoTLGfCr5//ZqY8xvjTEj8mEGxphPJPuwZjD/Dof8tHbydqAbgPNxbmjyGnC1tXbtIW3IATLGnA20AA9Za2dluz37wxgzAZhgrX3dGFMErADeN5L+LZJ3oiuw1rYYY/zAMuAT1tpXsty0/WKM+TRwIlBsrb0o2+3ZH8aYLcCJ1toR+51UY8yvgb9Za+9Lfksl31rbmO127a/k790dwCnW2q3Zbs9gGWMm4fw/fZy1tt0Ysxh4xlr7YHZbNjTGmFk4d9c8GYgAfwFuttZu7GufbIycB3M7UNez1r6Ic2X6iGWtfdda+3pyeR+wjp53f3M162hJrvqTrxF5IYUxZjLwb8B92W7LaGaMKQHOxvkWCtbayEgO5qTzgE0jKZgz+IC85D008oGdWW7P/jgWeNVa25a8i+YLwOX97ZCNcB7M7UDlEDPGTAPmAa9mtyVDlzwVvBLYA/zVWjvi+pD0X8DngUS2G3KALPB/xpgVybsCjjTTgVrggeQUw33GmIJsN+oAfQD4bbYbMVTW2h3A94FtwLtAk7X2/7Lbqv2yGjjLGDPGGJMP/Ctdb+7Vgy4IE4wxhcCTwCettc3Zbs9QWWvj1trjce5ed3LyFNKIYoy5CNhjrV2R7bYMgzOttSfgPMnu48kpoJHEB5wA/MxaOw9oBUbktTEAydPylwC/y3ZbhsoYU4ZzZnU6MBEoMMZ8KLutGjpr7TqcpzX+H84p7ZVAvL99shHOg7kdqBwiyXnaJ4FHrLW/z3Z7DkTy1ONS4MJst2U/nAFckpyvfQw41xjzcHabtH+Sox2stXuAp3CmskaSGqAm4wzMEzhhPVK9F3jdWrs72w3ZD/8CvGOtrbXWRoHfA6dnuU37xVr7K2vtfGvt2UADzrVXfcpGOA/mdqByCCQvpvoVsM5a+8Nst2d/GGMqjTGlyeU8nAsN12e3VUNnrf2CtXaytXYazv8Tz1trR9wIwRhTkLy4kOSp4PfgnNIbMay1u4DtxpjUQwrOA0bMRZK9uJoReEo7aRtwqjEmP/n76jyca2NGHGPM2OT7VJz55kf7q39In0oFfd8O9FC340AZY34LVAEVxpga4KvW2l9lt1VDdgbwYeDN5JwtwB3JO8KNFBOAXyevRvUAi621I/ZrSDlgHPBU8nHuPuBRa+1fstuk/bIIeCQ5gNgMXJ/l9uyX5B9I5wMfzXZb9oe19lVjzBPA60AM+Ccj905hTxpjxgBR4OMDXWSoO4SJiIi4jC4IExERcRmFs4iIiMsonEVERFxG4SwiIuIyCmcRERGXUTiLiIi4jMJZRETEZRTOIiIiLvP/AcalbWXRRdUUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_learning_curves(history):\n",
    "    pd.DataFrame(history.history).plot(figsize=(8, 5))\n",
    "    plt.grid(True)\n",
    "    plt.gca().set_ylim(0, 1)\n",
    "    plt.show()\n",
    "\n",
    "plot_learning_curves(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 62us/sample - loss: 0.3505 - accuracy: 0.8758\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.35052303292751313, 0.8758]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(x_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.saved_model.save(model, \"./keras_saved_graph\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:\n",
      "\n",
      "signature_def['__saved_model_init_op']:\n",
      "  The given SavedModel SignatureDef contains the following input(s):\n",
      "  The given SavedModel SignatureDef contains the following output(s):\n",
      "    outputs['__saved_model_init_op'] tensor_info:\n",
      "        dtype: DT_INVALID\n",
      "        shape: unknown_rank\n",
      "        name: NoOp\n",
      "  Method name is: \n",
      "\n",
      "signature_def['serving_default']:\n",
      "  The given SavedModel SignatureDef contains the following input(s):\n",
      "    inputs['flatten_input'] tensor_info:\n",
      "        dtype: DT_FLOAT\n",
      "        shape: (-1, 28, 28)\n",
      "        name: serving_default_flatten_input:0\n",
      "  The given SavedModel SignatureDef contains the following output(s):\n",
      "    outputs['dense_2'] tensor_info:\n",
      "        dtype: DT_FLOAT\n",
      "        shape: (-1, 10)\n",
      "        name: StatefulPartitionedCall:0\n",
      "  Method name is: tensorflow/serving/predict\n"
     ]
    }
   ],
   "source": [
    "!saved_model_cli show --dir ./keras_saved_graph --all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The given SavedModel SignatureDef contains the following input(s):\r\n",
      "  inputs['flatten_input'] tensor_info:\r\n",
      "      dtype: DT_FLOAT\r\n",
      "      shape: (-1, 28, 28)\r\n",
      "      name: serving_default_flatten_input:0\r\n",
      "The given SavedModel SignatureDef contains the following output(s):\r\n",
      "  outputs['dense_2'] tensor_info:\r\n",
      "      dtype: DT_FLOAT\r\n",
      "      shape: (-1, 10)\r\n",
      "      name: StatefulPartitionedCall:0\r\n",
      "Method name is: tensorflow/serving/predict\r\n"
     ]
    }
   ],
   "source": [
    "!saved_model_cli show --dir ./keras_saved_graph \\\n",
    "    --tag_set serve --signature_def serving_default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-07-14 18:29:21.575412: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0714 18:29:21.576680 140736297124800 deprecation.py:323] From /Users/zhangyx/workspace/environments/tf2_py3/lib/python3.7/site-packages/tensorflow/python/tools/saved_model_cli.py:339: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n",
      "W0714 18:29:21.633260 140736297124800 deprecation.py:323] From /Users/zhangyx/workspace/environments/tf2_py3/lib/python3.7/site-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "Result for output key dense_2:\n",
      "[[0.05805466 0.21746245 0.3026318  0.04621857 0.05800951 0.00982761\n",
      "  0.02890992 0.04335663 0.18267745 0.05285135]\n",
      " [0.05805466 0.21746245 0.3026318  0.04621857 0.05800951 0.00982761\n",
      "  0.02890992 0.04335663 0.18267745 0.05285135]]\n"
     ]
    }
   ],
   "source": [
    "!saved_model_cli run --dir ./keras_saved_graph --tag_set serve \\\n",
    "    --signature_def serving_default \\\n",
    "    --input_exprs 'flatten_input=np.ones((2, 28, 28))'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['serving_default']\n"
     ]
    }
   ],
   "source": [
    "loaded_saved_model = tf.saved_model.load('./keras_saved_graph')\n",
    "print(list(loaded_saved_model.signatures.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tensorflow.python.eager.function.ConcreteFunction object at 0x137bac9e8>\n"
     ]
    }
   ],
   "source": [
    "inference = loaded_saved_model.signatures['serving_default']\n",
    "print(inference)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dense_2': TensorSpec(shape=(None, 10), dtype=tf.float32, name='dense_2')}\n"
     ]
    }
   ],
   "source": [
    "print(inference.structured_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[7.4990585e-06 7.5836206e-06 5.3658046e-06 1.4856428e-06 2.8637976e-05\n",
      "  1.4206698e-02 1.7013150e-05 2.4653120e-02 2.4786242e-04 9.6082473e-01]], shape=(1, 10), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "results = inference(tf.constant(x_test_scaled[0:1]))\n",
    "print(results['dense_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
